# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import codecs
import json
from itemadapter import ItemAdapter

# useful for handling different item types with a single interface


class MyspiderPipeline:
    pass

class WangYiPipeline:
    """保存到JSON文件的管道"""
    def open_spider(self, spider):
        if spider.name == 'wangyi':
            """蜘蛛开启时创建文件"""
            self.file = open('wangyi_joblist.json', 'w', encoding='utf-8')
            self.file.write('[\n')  # JSON数组开始
            self.first_item = True

    def process_item(self, item, spider):
        if spider.name == 'wangyi':
            """处理每个item"""
            line = json.dumps(
                ItemAdapter(item).asdict(),
                ensure_ascii=False,
                indent=2
            )
            if not self.first_item:
                self.file.write(',\n')  # 除第一个item外，前面加逗号
            else:
                self.first_item = False
            self.file.write(line)

        return item

    def close_spider(self, spider):
        if spider.name == 'wangyi':
            """蜘蛛关闭时完成文件"""
            self.file.write('\n]')  # JSON数组结束
            self.file.close()
